{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants</th>\n",
       "      <th>Plasma_glucose_concentration</th>\n",
       "      <th>blood_pressure</th>\n",
       "      <th>Triceps_skin_fold_thickness</th>\n",
       "      <th>serum_insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Diabetes_pedigree_function</th>\n",
       "      <th>Age</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.848324</td>\n",
       "      <td>0.149641</td>\n",
       "      <td>0.907270</td>\n",
       "      <td>-0.692891</td>\n",
       "      <td>0.204013</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.123396</td>\n",
       "      <td>-0.160546</td>\n",
       "      <td>0.530902</td>\n",
       "      <td>-0.692891</td>\n",
       "      <td>-0.684422</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>1.943724</td>\n",
       "      <td>-0.263941</td>\n",
       "      <td>-1.288212</td>\n",
       "      <td>-0.692891</td>\n",
       "      <td>-1.103255</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-0.998208</td>\n",
       "      <td>-0.160546</td>\n",
       "      <td>0.154533</td>\n",
       "      <td>0.123302</td>\n",
       "      <td>-0.494043</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.504055</td>\n",
       "      <td>-1.504687</td>\n",
       "      <td>0.907270</td>\n",
       "      <td>0.765836</td>\n",
       "      <td>1.409746</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnants  Plasma_glucose_concentration  blood_pressure  \\\n",
       "0   0.639947                      0.848324        0.149641   \n",
       "1  -0.844885                     -1.123396       -0.160546   \n",
       "2   1.233880                      1.943724       -0.263941   \n",
       "3  -0.844885                     -0.998208       -0.160546   \n",
       "4  -1.141852                      0.504055       -1.504687   \n",
       "\n",
       "   Triceps_skin_fold_thickness  serum_insulin       BMI  \\\n",
       "0                     0.907270      -0.692891  0.204013   \n",
       "1                     0.530902      -0.692891 -0.684422   \n",
       "2                    -1.288212      -0.692891 -1.103255   \n",
       "3                     0.154533       0.123302 -0.494043   \n",
       "4                     0.907270       0.765836  1.409746   \n",
       "\n",
       "   Diabetes_pedigree_function       Age  Target  \n",
       "0                    0.468492  1.425995       1  \n",
       "1                   -0.365061 -0.190672       0  \n",
       "2                    0.604397 -0.105584       1  \n",
       "3                   -0.920763 -1.041549       0  \n",
       "4                    5.484909 -0.020496       1  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(\"FE_pima_indians_diabetes.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['Target']   \n",
    "X_train = train.drop([\"Target\"], axis=1)\n",
    "\n",
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# \t采用5折交叉验证的log似然损失和正确率（不包含调优）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [0.49442383 0.50834194 0.4830274  0.43615578 0.48793522]\n",
      "cv logloss is: 0.4819768328671518\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# 交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold,默认是分层抽样\n",
    "#作业要求，采用5折交叉验证\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()\n",
    "\n",
    "from sklearn.model_selection import cross_val_score #这个库主要为了得到分数\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "#%timeit loss_sparse = cross_val_score(lr, X_train_sparse, y_train, cv=3, scoring='neg_log_loss')\n",
    "print( 'logloss of each fold is: ',-loss)\n",
    "print('cv logloss is:', -loss.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy of each fold is:  [0.77272727 0.74675325 0.75324675 0.81699346 0.76470588]\n",
      "cv accuracy is: 0.7708853238265002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score #这个库主要为了得到分数\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='accuracy')\n",
    "#%timeit loss_sparse = cross_val_score(lr, X_train_sparse, y_train, cv=3, scoring='neg_log_loss')\n",
    "print( 'accuracy of each fold is: ',loss)\n",
    "print('cv accuracy is:', loss.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 采用5折交叉验证的log似然损失的超参数调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    # 使用GridSearchCV对log_loss优化的过程和结果如下："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#为了比较GridSearchCV和LogisticRegressionCV，两者用相同的交叉验证数据分割\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "fold = StratifiedKFold(n_splits=5, shuffle=True, random_state=777)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=777, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='liblinear',\n",
       "          tol=0.0001, verbose=0, warm_start=False),\n",
       "       fit_params=None, iid='warn', n_jobs=-1,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV #超参数调优的库\n",
    "from sklearn.linear_model import LogisticRegression #调优用的算法库\n",
    "\n",
    "#需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression(solver='liblinear')\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=fold, scoring='neg_log_loss',n_jobs = -1,)#负log似然损失\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4842142633700231\n",
      "{'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4lOW9//H3d7IQAoEECBAGFERQ9kQQQVChLqAiBHfaWm1ttVVqt2OrtkdbrdYef21tK1ZccDtW5KggKhU3EEGRzbDLIm5hEQj7mmXu3x8zaAwhM0lm8sxMPq/req7Ms818ngDz5X6W+zbnHCIiIjXxeR1ARETin4qFiIiEpWIhIiJhqViIiEhYKhYiIhKWioWIiISlYiEiImGpWIiISFgqFiIiElZqLN/czEYCfwdSgEedc/dWWf83YHhoNhNo65zLDq27GvhdaN0fnXNP1vRZbdq0cZ07d45iehGR5Ld48eLtzrnccNtZrLr7MLMUYC1wLlAMLATGOedWHWP7nwIFzrkfmFkrYBEwAHDAYqC/c27nsT5vwIABbtGiRVE+ChGR5GZmi51zA8JtF8vTUAOB9c65Dc65UmAyMKaG7ccBz4ZejwDecM7tCBWIN4CRMcwqIiI1iGWx8ANfVJovDi07ipkdD3QB3q7tviIiEnuxLBZWzbJjnfO6EnjeOVdRm33N7DozW2Rmi7Zt21bHmCIiEk4sL3AXA50qzXcENh1j2yuBG6vsO6zKvrOr7uScexh4GILXLOoeVUSSTVlZGcXFxRw6dMjrKHEhIyODjh07kpaWVqf9Y1ksFgLdzKwLsJFgQfh21Y3M7CQgB3i/0uKZwD1mlhOaPw+4NYZZRSTJFBcXk5WVRefOnTGr7mRF4+Gco6SkhOLiYrp06VKn94jZaSjnXDkwnuAX/2pginNupZndaWajK206DpjsKt2W5ZzbAdxFsOAsBO4MLRMRicihQ4do3bp1oy8UAGZG69at69XKiulzFs65GcCMKsturzL/+2PsOwmYFLNwIpL0VCi+Vt/fRaN/gjsQcNwzYzVf7DjgdRQR8dgVE9/nionvh9+wEWr0xeLTkv1MXvA5ox+Yy3vrt3sdR0SSSPPmzb96PXLkSLKzsxk1alS12954443k5+fTs2dPmjZtSn5+Pvn5+Tz//PO1+swlS5bw2muv1St3dRp9sTghtznTxw+ldfMmXDVpAU/M+4RYPdUuIo3XzTffzNNPP33M9RMmTKCoqIgZM2bQtWtXioqKKCoq4tJLL63V56hYxFDnNs2YesPpDD+pLb9/eRW/eWEZh8srwu8oIhKhs88+m6ysrDrtu27dOkaMGEH//v0588wzWbt2LQCTJ0+md+/e9OvXj+HDh3Pw4EHuvPNOnnnmmTq1SmoS0wvciSQrI42Hr+rP/W+u5R9vr2fd1n1M/G5/2rbI8DqaiNTTH15eyapNe8Jut2pzcJtIrlv07NCCOy7qVe9skbjuuut49NFH6dq1K/PmzWP8+PG8/vrr/OEPf2D27Nm0a9eOXbt20bRpU26//XZWrFjB/fffH9UMKhaV+HzGL887iZPzWvCrKUu56IG5TLxqAPmdsr2OJiKN1K5du5g/fz6XXHLJV8vKy8sBGDJkCN/73ve47LLLuPjii2OaQ8WiGhf0yaNLm2b86KlFXD7xff40tg+X9O/odSwRqaNIWwBHWhTPXT84lnFqxTlHmzZtKCoqOmrdI488wgcffMArr7xCv379WLZsWcxy6JrFMfTIa8H08UPpf1wOv/q/pdz1yirKKwJexxKRRiYnJ4e8vDymTp0KQCAQYOnSpQBs2LCBQYMGcdddd5GTk8PGjRvJyspi7969Uc+hYlGDVs3SeeragVxzemcem/sJ1zy+kF0HSr2OJSIJ6IwzzuCyyy7jrbfeomPHjsycOTPifSdPnsxDDz1Ev3796NWrF6+88goAv/jFL+jTpw99+vThnHPOoXfv3nzrW99i6dKlFBQURPUCd8wGP2posR78aMrCL/jdtBW0b5nBo1cPoHu7ut3VICINY/Xq1fTo0aNW+8Tjaahoqu53Eg+DHyWVy0/txLPXDeJgWQVjJ8xj5sotXkcSkSh77vrBSVso6kvFohb6H5/Dy+OHcmLb5lz/9GL+/uY6AoHkaJmJiNRExaKW2rfM4LnrB3PxKX7+9uZabnhmCfsPl3sdS0QkplQs6iAjLYW/XNaP/x7Vk9dXbeHiB9/j8xJ1RCgiyUvFoo7MjGuHduHJHwxky55DjJ4wl3nqiFBEkpSKRT2d0S2X6eOH0DarCd+btIBJc9URoUjCevzC4CRHUbGIguNbN+PFG4Zw9sltufOVVdz8/DIOlakjQpHG7kgX5UVFRQwePJhevXrRt29fnnvuuaO2jfcuytXdR5Q0b5LKQ9/tz9/fWsff31rH+q37mHhVf9qpI0KRRi8zM5OnnnqKbt26sWnTJvr378+IESPIzv6637kJEyYA8OmnnzJq1Khqu/eIxJIlS1ixYgUjR46MSvYj1LKIIp/P+MW53Xnou6ew9su9XPTPuSz5fKfXsUTEY927d6dbt24AdOjQgbZt27Jt27aI91cX5UlqZO88Ooc6Irxy4nzuHtubywZ08jqWSOP1n1tgy/Lw220JdcQXyXWL9n3g/HtrHWXBggWUlpbStWvXiPdRF+VJ7OT2LZh+41DGP7uEm59fxqrNe/jtBT1ITVFjTqSx2rx5M1dddRVPPvkkPl9k3wXqorwRyGmWzpPfH8g9Mz5i0rxPWPvlXh4Ydwo5zdK9jibSuETaAjjSovj+q1GPsGfPHi688EL++Mc/MmjQoIj3UxfljURqio/bL+rJfZf2ZeEnOxk9YS4fbQk/YpeIJI/S0lLGjh37VSugNtRFeSNz2YBOPHf9IA6XBbj4wfd4bcVmryOJSAOZMmUKc+bM4Yknnvjqltja3O2kLsqjKNZdlEfLl3sOcf3Tiyn6Yhc3nd2Nn5/dDZ/PvI4lknTq0kV5LE9DxYP6dFGuaxYNrF2LDCZfN4jfTVvBP95ax+rNe/jbFfk0b6I/ChHPJWmRiAadhvJARloK913alzsu6snbH23l4gfn8VnJfq9jiYgck4qFR8yM7w/pwlM/GMjWvYcZ/cA83l0X+UM6IhJespxmj4b6/i5ULDw25MQ2TL9xKO1bZHD1pAU8+u4G/QUXiYKMjAxKSkr074lgoSgpKSEjo+7dD+lEeRw4rnUmL95wOr+aspQ/vrqaVZv3cM/YPmSkpXgdTSRhdezYkeLi4lp1q5HMMjIy6NixY533V7GIE82apPLgd07hgVnr+esba/l46z4mXjWA9i3VEaFIXaSlpdGlSxevYyQNnYaKIz6fcdPZ3Zh4VX/Wb93HRQ/MZfFn6ohQRLynYhGHRvRqz9Qbh5CZnsK4h+czZeEXXkcSkUZOxSJOdW+XxUs3DuG0E1rx6xeW8fvpKymrCHgdS0QaKRWLOJadmc7j15zKD4d24Yn3PuV7jy1gx/5Sr2OJSCOkYhHnUlN8/G5UT/5yWT8Wf76T0Q/MZdUmdUQoIg0rpsXCzEaa2RozW29mtxxjm8vNbJWZrTSzf1daXmFmRaFpeixzJoJL+ndkyvWDKasIcMm/3mPGcnVEKCINJ2bFwsxSgAnA+UBPYJyZ9ayyTTfgVmCIc64X8PNKqw865/JD0+hY5Uwk+Z2yeXn8UHrkZXHDM0v4y+trCAT0wJGIxF4sWxYDgfXOuQ3OuVJgMjCmyjY/AiY453YCOOe2xjBPUmjbIoNnrxvE5QM68s+313Pd04vZe6jM61gikuRiWSz8QOV7PotDyyrrDnQ3s3lmNt/MRlZal2Fmi0LLC2OYM+E0SU3hz5f05Q+jezFrzVbGPvgen2xXR4QiEjuxLBbVDdJQ9ZxJKtANGAaMAx41s+zQuuNCfax/G7jfzI4a3dzMrgsVlEWN7ZF+M+Pq0zvz9LUDKdl3mDEPzOWdtY3rdyAiDSeWxaIY6FRpviOwqZptXnLOlTnnPgHWECweOOc2hX5uAGYDBVU/wDn3sHNugHNuQG5ubvSPIAGc3rUN08cPpUN2U66etIAz/+dtryOJSBKKZbFYCHQzsy5mlg5cCVS9q2kaMBzAzNoQPC21wcxyzKxJpeVDgFUxzJrQOrXK5IWfnE5OZhqf7zjIh5+rixARia6YFQvnXDkwHpgJrAamOOdWmtmdZnbk7qaZQImZrQJmATc750qAHsAiM1saWn6vc07FogbNmqRyQm4zzGDahxu9jiMiSUZjcCeZG/+9hPc/LuGD284mLUXPXIpIzSIdg1vfJklmbL6fHftLNeqeiESVikWSObN7LjmZaUz9sOq9BCIidadikWTSU32M6tuB11du0cN6IhI1KhZJqLDAz+HyAK+t2OJ1FBFJEioWAI9fGJySxCnHZXNcq0ymFemuKBGJDhWLQAXs3woVyTNOhJlRWODnvY9L2LL7kNdxRCQJqFjs+gy2r4XdxV4niaqxBX6cg+lL1boQkfpTsWh1AjRrC/u2wJ7kGSOiS5tm5HfK1l1RIhIVKhYALY8D52DuX71OElVjC/ys3ryHj7ZoZD0RqR8VC4Dr3oJTvgeLn4BdX4TdPFGM6ptHis+YptaFiNSTisURZ94cbF28+xevk0RN6+ZNOKt7Li8VbdSIeiJSLyoWR2R3gv5Xw4dPw85PvU4TNYUFfjbvPsQHn+zwOoqIJDAVi8rO+BVYCsy5z+skUXNuj3Y0S09RT7QiUi8qFpW16AADfgBFz0LJx16niYqm6SmM7J3HjOWbOVRW4XUcEUlQKhZVDf0FpKTDO//jdZKoGVvgZ+/hct7+aKvXUUQkQalYVJXVDgb+EJZPgW1rvU4TFYO7tqZtVhOm6lSUiNSRikV1hvwcUpvCO/d6nSQqUnzGmPwOzF6zlZ37k6dbExFpOCoW1WnWBk67Hla8CF8mx2iuhQV+yiocry5PnqfURaThqFgcy+k/hfTmMPtPXieJip55LejerrnuihKROlGxOJbMVjD4Blg9HTYv8zpNvR3piXbRZzv5vOSA13FEJMGoWNRk0A3QpGXStC7G5PsBeEnjXIhILalY1KRpNpw+HtbMgI1LvE5Tb/7sppzWpRVTizbinLr/EJHIqViEc9qPoWkOzLrH6yRRMbbAz4Zt+1m+cbfXUUQkgahYhJPRAk6/Cda/AV8s8DpNvZ3fJ4/0FJ+euRCRWlGxiMTA6yCzTVK0Llo2TePsHm15eekmyisCXscRkQShYhGJJs1h6M9hwyz47D2v09RbYYGf7ftKmbt+u9dRRCRBqFhEasC10LxdUrQuhp2US8umaXrmQkQipmIRqfRMGPpL+PRd2PCO12nqpUlqChf2zWPmyi/Zf7jc6zgikgBULGqj/zWQ1SHYukjwW0/HFvg5WFbB66u2eB1FRBKAikVtpGXAmb+CL+bDx295naZe+h+XQ8ecpkzV+NwiEgEVi9oquApadkr41oXPZxTm+5m7bhtb9x7yOo6IxDkVi9pKbQJn3gwbF8PamV6nqZfCgg4EHLy8VD3RikjNVCzqIv/bkNMZZt2d0K2LE9tm0cffUndFiUhYKhZ1kZIGZ/0GtiyDj171Ok29FBb4Wb5xN+u37vU6iojEMRWLuupzObQ+MXjtIpC4T0Jf1C8Pn8E0XegWkRrEtFiY2UgzW2Nm683slmNsc7mZrTKzlWb270rLrzazdaHp6ljmrJOUVDjrFti6Ela/5HWaOmublcHQbrlM/XAjgUDinlITkdiKWbEwsxRgAnA+0BMYZ2Y9q2zTDbgVGOKc6wX8PLS8FXAHcBowELjDzHJilbXOel8MbU6CWX+CQIXXaepsbEEHNu46yKLPdnodRUTiVCxbFgOB9c65Dc65UmAyMKbKNj8CJjjndgI457aGlo8A3nDO7QitewMYGcOsdeNLgWG3wPY1wfG6E9R5PdvTNC1FPdGKyDHFslj4gS8qzReHllXWHehuZvPMbL6ZjazFvvGhZyG07RUcTa8iMbvOaNYklRG92vHqsk0cLk/cFpKIxE4si4VVs6zqSfFUoBswDBgHPGpm2RHui5ldZ2aLzGzRtm3b6hm3jnw+GH4r7PgYlk/xJkMUjD2lI3sOlTPrI49+jyIS12pdLMzMZ2YtIti0GOhUab4jUPWWm2LgJedcmXPuE2ANweIRyb445x52zg1wzg3Izc2tzWFE18mjoH1feOfPUFHmXY56GNK1NW2aN9EzFyJSrYiKhZn928xamFkzYBWwxsxuDrPbQqCbmXUxs3TgSmB6lW2mAcNDn9GG4GmpDcBM4Dwzywld2D4vtCw+mcHw38LOT6Ho32E3j0epKT5G9+vA2x9tZfeBxCx4IhI7kbYsejrn9gCFwAzgOOCqmnZwzpUD4wl+ya8GpjjnVprZnWY2OrTZTKDEzFYBs4CbnXMlzrkdwF0EC85C4M7QsvjVfQT4+8Oc+6C81Os0dTK2wE9pRYAZK9T9h4h8U6TFIs3M0ggWi5ecc2VUcw2hKufcDOdcd+dcV+fc3aFltzvnpodeO+fcL51zPZ1zfZxzkyvtO8k5d2Joerz2h9bAzGD4bbD7C/jwKa/T1Elvfwu65jbTXVEicpRIi8VE4FOgGTDHzI4H9sQqVMLqejZ0GgRz/gJlideTq5kxtsDPgk92ULzzgNdxRCSORFQsnHP/cM75nXMXhFoDnxG61iCVHGld7N0Ei5/wOk2djMkP3qH8UpG6/xCRr0V6gftnoQvcZmaPmdkS4FsxzpaYupwJxw+FuX+F0sT733mnVpmc2jmHqR9uxCVwj7oiEl2Rnob6QegC93lALvB94N6YpUpkR1oX+76ERY95naZOCgv8rN+6j5WbdKZRRIIiLRZHHpK7AHjcObeU6h+cE4DOQ+CEYTD3fji8z+s0tXZhnzzSUkzPXIjIVyItFovN7HWCxWKmmWUBidsvd0MY/ls4sB0WPuJ1klrLzkxn+ElteWnpJirUE62IEHmxuBa4BTjVOXcASCd4KkqOpdNAOPFcmPd3OJR4p3PGFvjZtvcw73283esoIhIHIr0bKkCwy43fmdn/A053zi2LabJkMPw2OLgTPpjodZJaG35yW7IyUvXMhYgAkd8NdS/wM4JdfawCbjKzP8UyWFLwnwInXQDv/xMO7vI6Ta1kpKVwYZ88Zq7YwoHSxOxNV0SiJ9LTUBcA54aeqp5EcGyJC2MXK4kMuxUO7Yb5D3qdpNYKC/zsL63gjVVfeh1FRDxWm15nsyu9bhntIEkrry/0GA3vPwgH4rt7q6oGdm5Fh5YZuitKRCIuFn8CPjSzJ8zsSWAxcE/sYiWZYbdC6T54759eJ6kVn88YU+BnzrrtbN932Os4IuKhSC9wPwsMAl4MTYMrd/onYbTrGRyv+4OJsD+x7i4aW+CnIuB4Zam6/xBpzGosFmZ2ypEJyCM4KNEXQIfQMonUWbdA+UGYd7/XSWqle7sseua1YKr6ihJp1FLDrP9LDesc6h8qcrndoc/lsOBRGPxTyGrndaKIjS3wc/eM1WzYto8Tcpt7HUdEPFBjy8I5N7yGSYWits76NVSUwty/eZ2kVkbnd8AMpql1IdJoRfqcxcXVTGebWdtYB0wqrbtC/jhYNAn2JM4Xb7sWGQzp2oZp6olWpNGqTXcfjwLfCU2PAL8E5plZjcOrShVn3gyuAt6t6Qxf/Cks8PP5jgMs+TyxHi4UkeiItFgEgB7OuUucc5cAPYHDwGnAb2IVLinldIaC78LiJ2HXF16nidiIXu3ISPPpmQuRRirSYtHZOVf5Md6tQHfn3A6gLPqxktwZ/xUc92LOfV4niVhWRhrn9mzPK8s2UVquDodFGptIi8W7ZvaKmV1tZlcD0wmOxd0M0HmJ2sruBKdcDUXPwI5PvE4TsbEFHdh5oIw5a7d5HUVEGlikxeJG4HEgHygAngRudM7td85pLO66OONXYCkJ1bo4o1surZqlM7VIp6JEGptIn+B2wFzgbeBNYI7TbTH10yIPTr0Wlj4LJR97nSYiaSk+Luqbx5urvmTPIZ19FGlMIr119nJgAXApcDnwgZldGstgjcLQX0BKE3jnz14niVhhgZ/D5QFeW77F6ygi0oAiPQ31W4Kj5F3tnPseMBD479jFaiSat4WBP4JlU2DbGq/TRCS/UzadW2dqUCSRRibSYuFzzm2tNF9Si32lJkN+BmmZMPter5NExMwoLPAz/5MSNu066HUcEWkgkX7hv2ZmM83sGjO7BngVmBG7WI1IszYw6Mew8kX4cqXXaSJSmO/HOZiunmhFGo1IL3DfDDwM9AX6AQ875/QwXrQMHg9NWsDsxBiptnObZhQcl60H9EQakYhPJTnnXnDO/dI59wvn3NRYhmp0MlvBoBtg9cuweanXaSJycYGfj7bsZfXmPV5HEZEGEG48i71mtqeaaa+Z6Vsimgb9BDJawqzEaF1c2LcDqT5T60KkkQjXRXmWc65FNVOWc65FQ4VsFJpmw+k/hbX/geLFXqcJq1WzdIadlMtLRZuoCOiRG5Fkpzua4slpP4amrWB2YgxvXljgZ8ueQ3ywocTrKCISYyoW8aRJVvBW2vVvwucfeJ0mrHN6tKN5k1Q9cyHSCKhYxJuBP4LMNjDrbq+ThJWRlsL5vdvznxVbOFRW4XUcEYkhFYt4k94s2A3IJ+/Ap3O9ThPW2AI/+w6X8+bqL8NvLCIJS8UiHp16LTRvB7PugTjvr/G0E1rTvkWG7ooSSXIxLRZmNtLM1pjZejO7pZr115jZNjMrCk0/rLSuotLy6bHMGXfSmga7MP9sXrCFEcdSfMaY/A7MXrONHftLvY4jIjESs2JhZinABOB8gsOwjjOzntVs+pxzLj80PVpp+cFKy0fHKmfcOuVqaOGHt++O+9ZFYYGf8oDj1WXq/kMkWcWyZTEQWO+c2+CcKwUmA2Ni+HnJJS0j2LooXgDr3/I6TY165LXg5PZZuitKJInFslj4gS8qzReHllV1iZktM7PnzaxTpeUZZrbIzOabWWEMc8avgqug5XEw648J0bpY8vkuPivZ73UUEYmBWBYLq2ZZ1W+8l4HOzrm+BEfge7LSuuOccwOAbwP3m1nXoz7A7LpQQVm0bVsSjgudmg5n3QybPoS1r3mdpkaj+3XADKZ9qFNRIskolsWiGKjcUugIfOObxDlX4pw7HJp9BOhfad2m0M8NwGyCY39TZf+HnXMDnHMDcnNzo5s+XvQbBzldgs9dxHHrokN2UwZ1ac20oo1oxF2R5BPLYrEQ6GZmXcwsHbgS+MZdTWaWV2l2NLA6tDzHzJqEXrcBhgCrYpg1fqWkwVm/gS3Lg73SxrGxBX4+2b6fpcW7vY4iIlEWs2LhnCsHxgMzCRaBKc65lWZ2p5kdubvpJjNbaWZLgZuAa0LLewCLQstnAfc65xpnsQDocxm0PjE43kUg4HWaYxrZpz3pqT49cyGShCxZThkMGDDALVq0yOsYsbP8eXjhWrh0EvS+xOs0x3TjM0uYv6GE+bedTVqKnvkUiXdmtjh0fbhG+tecKHqNhdyTg2N1B+K3H6bCAj8l+0uZu26711FEJIpULBKFLwWG3Qrb1wZbGXHqrO65ZGem6ZkLkSSjYpFIeoyGdr3hnXuhotzrNNVKT/Uxqm8er6/awr7D8ZlRRGpPxSKR+Hww/DbYsQGWTfY6zTGNLfBzqCzAzBVbvI4iIlGiYpFoTroA8vrBO3+GijKv01TrlONy6NSqKdOKdCpKJFmoWCQaMxj+W9j1ORQ943WaapkZY/P9zFu/nS/3HPI6johEgYpFIup2HvgHwDv3Qfnh8Nt7YEyBn4CDl5eq+w+RZKBikYjMgtcu9hTDkqe8TlOtrrnN6dexpe6KEkkSKhaJquu34LjB8O5foOyg12mqVVjgZ+WmPaz9cq/XUUSknlQsEtWR1sXezbD4Ca/TVGtU3w6k+Ezdf4gkARWLRNblTOh8Brz7Vyg94HWao+RmNeGMbm14qWgTgUBydCsj0lipWCS64b+F/Vth4aPw+IXBKY6MLfCzcddBFny6w+soIlIPKhaJ7vjBwesX8+6HQPw9MX1uz3ZkpqfoVJRIglOxSAbDboMDJcHrF3EmMz2Vkb3a8+ryzRwqi98OEEWkZioWyaDTqcFnL/ZsjMvWRWGBn72Hypn10Vavo4hIHalYJIvhtwULxZ74ewju9K6tyc1qomcuRBKYikWy6FAATVsFWxebPvQ6zTekpvgY068Ds9ZsZdeBUq/jiEgdqFgkk5zOwXEvHj0n+LBeHA2SVFjgp6zC8ery+LuuIiLhqVgkk7RMyCuAk0fBW3fCkxcFOxyMA706tKBb2+a6K0okQalYJJuUNLjsCSh8CDYvg38NhWX/53UqzIzCAj8LP93JFzvi7wFCEamZikUyMoP8cfCTudD2ZHjxh/D8tXBwl6exxuR3AOAljXMhknBULJJZTme4ZkbwKe+VU+GhofDpPM/idMzJZGCXVkz9cCPOqfsPkUSiYpHsUlLhrF/Dta8HT1E9cSG8+Xso9+aupLEFfj7etp8VG/d48vkiUjcqFsnk+68Gp+p0HADXvwsF34W5f4PHzoFtaxs2H3BB7zzSU3x65kIkwahYNCZNmsOYB+CK/4VdX8DEM2HhY9CAp4RaZqbxrZPbMn3pJsorAg32uSJSPyoWjVGPi+An7wU7IXz1l/DslbBvW4N9fGGBn+37DjPv45IG+0wRqR8Vi8aqRR585wUY+Wf4eBb8azCsfb1BPnr4ybm0yEjVMxciCUTFojHz+WDQj+G62dC8Hfz7Mnj1VzEfSKlJagoX9u3Aayu2sP9w/HV8KCJHU7EQaNcTfvgWDB4fHETp4bNg89KYfuTYAj8Hyyp4Y9WXMf0cEYkOFQsJSsuAEXfDVdPg8F545OzgXVMx6l9qwPE5+LOb6q4okQShYiHf1HV48OL3SecHn8d4cnTwzqko8/mMwoIOvLtuG9v2Ho76+4tIdKlYyNEyW8HlT8GYB2FzEfxrCCx/PuofU5jvJ+Dg5aXxNwaHiHyTioVUzwwKvgM/fhdyu8ML18ILP4JDu6P2Ed3aZdHb34Jp6itKJO6pWEjNWp0A338Nht0KK14I9mL72XtRe/v0bGimAAAM4ElEQVTCfD/Linezfuu+qL2niESfioWEl5IKw26BH8wM3m77xIXB8TIqyur91qP7dcBn6olWJN7FtFiY2UgzW2Nm683slmrWX2Nm28ysKDT9sNK6q81sXWi6OpY5JUKdToUfz4X8bwdH4nvsXNi+vl5v2bZFBkNObKOeaEXiXMyKhZmlABOA84GewDgz61nNps855/JD06OhfVsBdwCnAQOBO8wsJ1ZZpRaaZMGYCXD507DzU5h4Bix6vF79S40t8FO88yCLP9sZvZwiElWxbFkMBNY75zY450qBycCYCPcdAbzhnNvhnNsJvAGMjFFOqYueo4O32HYaCK/8HCZ/G/Zvr9NbjejVnqZpKXrmQiSOxbJY+IHKN+gXh5ZVdYmZLTOz582sUy33FS+16ADfnQoj/gTr34QHB8O6N2r9Ns2apHJer3a8smwzpeXqiVYkHsWyWFg1y6qeq3gZ6Oyc6wu8CTxZi30xs+vMbJGZLdq2reF6TZVKfD4YfEOwf6lmbeCZS2HGzVB2sFZvU1jgZ/fBMmav2RqTmCJSP7EsFsVAp0rzHYFvPH3lnCtxzh15fPcRoH+k+4b2f9g5N8A5NyA3NzdqwaUO2vWCH82CQTfAgofh4WGweVnEu59xYhtaN0vXMxdxbuU9Q1l5z1CvY9RbshwHNNyxxLJYLAS6mVkXM0sHrgSmV97AzPIqzY4GVodezwTOM7Oc0IXt80LLJJ6lZcDIP8F3X4SDu+CRb8G8f0Ag/Kml1BQfF/XrwJurt7L7YBlXTHyfKya+3wChYy+Zvpik8YpZsXDOlQPjCX7JrwamOOdWmtmdZjY6tNlNZrbSzJYCNwHXhPbdAdxFsOAsBO4MLZNEcOLZwYvf3UfAG/8NT42G3cVhdxtb4Ke0PMBrKzY3QEgRqY3UWL65c24GMKPKstsrvb4VuPUY+04CJsUyn8RQs9bB4Vs//F/4z2/gX6fDqPuh98XH3KVvx5ac0KYZLy7ZyO0lN4eWzm2YvCJSIz3BLbFjBqdcFexfqnU3eP778OL1cGjPMTY3Cgv8fPDJDrYGWjRwWBGpiYqFxF7rrvCD1+Cs38DyKfDQUPh8frWbFuYH75CeXVbd85si4hUVC2kYKWkw/LZg/1Jm8Pj58PYfj+pf6rjWmfQ/PodZZb3r81C4iESZioU0rE4Dg/1L9RsHc+6DSSOg5ONvbFJY4OfzQC4bAm09CikiValYSMNrkgWFD8JlTwYLxUNnwOInv+pfalSfPFKpYEbpKWzcdZB9h8vVyaCIx2J6N5RIjXoVQsdTYdpP4OWbYN3rcNE/yGnWmlNT1/NaWQGv3fs2AKk+IzszjZZN08jOTCe7aRotM9PIbppOdmbaUeuyQ+uyMlLx+arrEEBEakPFQrzV0g9XTYP5E4JjZPxrMBQ+yE0Z/2F4xQpaXvB7dh0oY9fBMnYdKGP3wVJ2HShj8+5DfLRlL7sOlLK/tOKYb29GsIg0TaNlZvpXr4PFJLjsq/nMNFqGik/LpmmkpajhLXKEioV4z+eD038KJwwLDt36v5fQ3dealqn76XnqcWF3Ly0PsPtgWWgKFpMjBWb3gdKvCs2R+c9K9rPrQBl7DpXVeBG9eZPUUGvl65ZKyyNFpuk3i0t2pVZORlpK1H41OBeaAkDopwt8vewby121y12ggoBzVFQceR2goqICVxGgwrngstA2gYoKXCBAIBAg4ALBeRcgEAhut6O8CQ5j/cLXv+6szTlc1a7bQr9YV82yb6745p7BTdxR21d+t2/+mYV/Tyq955E1W8sycRiB2S8Qa7E+gbq1rBkpOHrF+HMsWc4FDxgwwC1atMjrGFJfZQfhzd/DBw8RwPClNa20ssrpJKt6eskiXBf8ceQ7JIDhXPBrJPjTCLjg66/WueAXTcB9vW1w96/f13319obPghl8ZlBRigGWkoYRwBf8JMwFMFxwcg6j0jyOFNQDr0RmdeA4ety5vE77mtli59yAcNupZSHxJa0pnP9nPl08k+aBvbQ5dVxweU3/qTlqXfX/y626zkLrvtkOqPw/1erf1zkor6jgcEWA0vLgdLisIvizIviztOzr1wf37iSAj5QmLXD4MDOc+cBCrzGoPF/pNfhwR177fJj5gsmPvDYLbVtlG7PgMl8KcOS1D/NZaH0K5jN8oeVH3sMX2t98hvlSKi0z8KWwfc5jALQb9qPQ7zBYLF2V4mzwVcH+5qqvZ75ebtUstypbB2dcNcvtqP8YEDr+Su9ZJcjm/9wHQIcLfn30vglm04z/ITXV0SPGn6NiIXFpv685+33NaXPeH72OchQD0kJTJI50Itjrthlhtox/K+f/DYBew8Z6nKR+Am/+AYCTTz3H4yT1V/HG7xvkc3QFT0REwlKxEBGRsFQsREQkLF2zkLjUK6+l1xFEpBK1LEREJCwVCxERCUvFQkREwlKxEBGRsHSBW+LT91/1OoGIVKKWhYiIhKWOBEVEGrFIOxJUy0JERMJSsRARkbBULEREJCwVCxERCUvFQkREwlKxEBGRsFQsREQkLBULEREJS8VCRETCSponuM1sG/BZPd6iDbA9SnG8lCzHATqWeJUsx5IsxwH1O5bjnXO54TZKmmJRX2a2KJJH3uNdshwH6FjiVbIcS7IcBzTMseg0lIiIhKViISIiYalYfO1hrwNESbIcB+hY4lWyHEuyHAc0wLHomoWIiISlloWIiISlYhFiZneZ2TIzKzKz182sg9eZ6srM7jOzj0LHM9XMsr3OVFdmdpmZrTSzgJkl3J0rZjbSzNaY2Xozu8XrPPVhZpPMbKuZrfA6S32YWSczm2Vmq0N/t37mdaa6MrMMM1tgZktDx/KHmH2WTkMFmVkL59ye0OubgJ7OuR97HKtOzOw84G3nXLmZ/RnAOfcbj2PViZn1AALAROC/nHMJMxyimaUAa4FzgWJgITDOObfK02B1ZGZnAvuAp5xzvb3OU1dmlgfkOeeWmFkWsBgoTMQ/FzMzoJlzbp+ZpQFzgZ855+ZH+7PUsgg5UihCmgEJW0Wdc68758pDs/OBjl7mqQ/n3Grn3Bqvc9TRQGC9c26Dc64UmAyM8ThTnTnn5gA7vM5RX865zc65JaHXe4HVgN/bVHXjgvaFZtNCU0y+u1QsKjGzu83sC+A7wO1e54mSHwD/8TpEI+UHvqg0X0yCfiklKzPrDBQAH3ibpO7MLMXMioCtwBvOuZgcS6MqFmb2ppmtqGYaA+Cc+61zrhPwDDDe27Q1C3csoW1+C5QTPJ64FcmxJCirZlnCtliTjZk1B14Afl7lzEJCcc5VOOfyCZ5BGGhmMTlFmBqLN41XzrlzItz038CrwB0xjFMv4Y7FzK4GRgFnuzi/MFWLP5dEUwx0qjTfEdjkURapJHR+/wXgGefci17niQbn3C4zmw2MBKJ+E0KjalnUxMy6VZodDXzkVZb6MrORwG+A0c65A17nacQWAt3MrIuZpQNXAtM9ztTohS4KPwasds791es89WFmuUfudjSzpsA5xOi7S3dDhZjZC8BJBO+8+Qz4sXNuo7ep6sbM1gNNgJLQovkJfGfXWOCfQC6wCyhyzo3wNlXkzOwC4H4gBZjknLvb40h1ZmbPAsMI9nD6JXCHc+4xT0PVgZkNBd4FlhP89w5wm3Nuhnep6sbM+gJPEvz75QOmOOfujMlnqViIiEg4Og0lIiJhqViIiEhYKhYiIhKWioWIiISlYiEiImGpWIjUgpntC79Vjfs/b2YnhF43N7OJZvZxqMfQOWZ2mpmlh143qodmJb6pWIg0EDPrBaQ45zaEFj1KsGO+bs65XsA1QJtQp4NvAVd4ElSkGioWInVgQfeF+rBabmZXhJb7zOzBUEvhFTObYWaXhnb7DvBSaLuuwGnA75xzAYBQ77SvhradFtpeJC6omStSNxcD+UA/gk80LzSzOcAQoDPQB2hLsPvrSaF9hgDPhl73Ivg0esUx3n8FcGpMkovUgVoWInUzFHg21OPnl8A7BL/chwL/55wLOOe2ALMq7ZMHbIvkzUNFpDQ0OI+I51QsROqmuu7Ha1oOcBDICL1eCfQzs5r+DTYBDtUhm0jUqViI1M0c4IrQwDO5wJnAAoLDWl4SunbRjmDHe0esBk4EcM59DCwC/hDqBRUz63ZkDA8zaw1sc86VNdQBidRExUKkbqYCy4ClwNvAr0OnnV4gOI7FCoLjhn8A7A7t8yrfLB4/BNoD681sOfAIX493MRxIuF5QJXmp11mRKDOz5s65faHWwQJgiHNuS2i8gVmh+WNd2D7yHi8Ctybw+OOSZHQ3lEj0vRIakCYduCvU4sA5d9DM7iA4Dvfnx9o5NFDSNBUKiSdqWYiISFi6ZiEiImGpWIiISFgqFiIiEpaKhYiIhKViISIiYalYiIhIWP8f3CtwPnClxjkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    #plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'logloss' )\n",
    "plt.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_best = grid.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    # 使用LogisticRegressionCV的过程和结果如下："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.001, 0.01, 0.1, 1, 10, 100, 1000],\n",
       "           class_weight=None,\n",
       "           cv=StratifiedKFold(n_splits=5, random_state=777, shuffle=True),\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=-1, penalty='l1',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "Cs = [1e-3, 1e-2, 1e-1, 1, 10, 100, 1000]\n",
    "#nCs = 9  #Cs values are chosen in a logarithmic scale between 1e-4 and 1e4.\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L1正则 --> 可选用saga优化求解器(0.19版本新功能)\n",
    "#LogisticRegressionCV比GridSearchCV快\n",
    "\n",
    "##  multi_class:分类方式选择参数，有\"ovr(默认)\"和\"multinomial\"两个值可选择，在二元逻辑回归中无区别\n",
    "##  cv:几折交叉验证\n",
    "##  solver:优化算法选择参数，当penalty为\"l1\"时，参数只能是\"liblinear(坐标轴下降法)\"\n",
    "##  \"lbfgs\"和\"cg\"都是关于目标函数的二阶泰勒展开\n",
    "##  当penalty为\"l2\"时，参数可以是\"lbfgs(拟牛顿法)\",\"newton_cg(牛顿法变种)\",\"seg(minibactch随机平均梯度下降)\"\n",
    "##  维度<10000时，选择\"lbfgs\"法，维度>10000时，选择\"cs\"法比较好，显卡计算的时候，lbfgs\"和\"cs\"都比\"seg\"快\n",
    "##  penalty:正则化选择参数，用于解决过拟合，可选\"l1\",\"l2\"\n",
    "\n",
    "lrcv_L1 = LogisticRegressionCV(Cs=Cs, cv = fold, scoring='neg_log_loss', penalty='l1', solver='liblinear', multi_class='ovr',n_jobs=-1)\n",
    "lrcv_L1.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.69314718 -0.64739728 -0.49389436 -0.49496646 -0.49618668 -0.49633143\n",
      "  -0.49633823]\n",
      " [-0.69314718 -0.6445387  -0.47101363 -0.47224691 -0.47360533 -0.47375456\n",
      "  -0.47376737]\n",
      " [-0.69314718 -0.64643399 -0.4923785  -0.48717398 -0.48793896 -0.48802734\n",
      "  -0.48803694]\n",
      " [-0.69314718 -0.64520245 -0.48498644 -0.46617388 -0.46455415 -0.46440024\n",
      "  -0.46438508]\n",
      " [-0.69314718 -0.65066481 -0.49883114 -0.50050282 -0.50183858 -0.50198397\n",
      "  -0.50200225]]\n"
     ]
    }
   ],
   "source": [
    "print(lrcv_L1.scores_[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl0XOWZ5/HvU6XNu2RbtkvGG2AWLzJgsZrmQAjgYAUIJCwhBJoJbjohJGdOZ0LSPc00ZGFOn2QynQECARLoTnBIWAK2wTFLMCEslgm2kQ3GmMXCm7zjVZb0zB91BYUsuUrL1a3l9zmnjuu+dW/Vc8HUj/vet97X3B0REZFDiUVdgIiIZD+FhYiIpKWwEBGRtBQWIiKSlsJCRETSUliIiEhaCgsREUlLYSEiImkpLEREJK2iMN/czGYC/xeIA/e4+23tXv8/wFnBZn9ghLuXB69dDfxL8NoP3P3+Q33W8OHDffz48b1YvYhI/luyZMlmd69Mt5+FNd2HmcWBVcA5QAOwGLjC3Vd0sv83gePd/VozGwrUATWAA0uA6e6+rbPPq6mp8bq6ul4+CxGR/GZmS9y9Jt1+YXZDnQSsdvc17t4EzAEuPMT+VwAPBs/PAxa6+9YgIBYCM0OsVUREDiHMsBgNrE3ZbgjaDmJm44AJwLNdPVZERMIXZlhYB22d9XldDvzB3Vu6cqyZzTazOjOra2xs7GaZIiKSTphh0QCMSdk+DFjXyb6X80kXVMbHuvvd7l7j7jWVlWnvz4iISDeFGRaLgYlmNsHMSkgGwuPtdzKzo4EK4KWU5gXAuWZWYWYVwLlBm4iIRCC0obPu3mxmN5D8ko8D97l7vZndAtS5e1twXAHM8ZRhWe6+1cxuJRk4ALe4+9awahURkUMLbehsX9PQWRGRrsuGobM5oaXV+dH8lTRs2xN1KSIiWavgw+L9LbuZ8+oHXHT7X1m6dnvU5YiIZKWCD4vDKwfyyNdPo6w4xmV3v8Sf6jdEXZKISNYp+LAAOHLEIB79+gyOHjWYf/ivJdz7l3fJl3s5IiK9QWERqBxUypzrTuG8SaO4de4K/tfj9bS0KjBEREBh8Sn9SuLcceUJzD7jcO5/6X1mP1DH7v3NUZclIhI5hUU7sZjx/fOP5daLpvDcW5u49K6X2LhzX9RliYhESmHRiatOGce9V5/Ie5t384XbX+TNDTujLklEJDIKi0M465gRPHT9qbS488U7X2LRKk1WKCKFSWGRxuSqITz2jRmMGdqfv//1Yh589YOoSxIR6XMKiwwkhvTj99efyulHDud7jyzntiffpFUjpUSkgCgsMjSwtIh7r67hypPH8ovn3+Gbc/7GvgMt6Q8UEckDoc06m4+K4jF+cNEUxg3rz4/mv8n67Xv55VdrGDawNOrSRERCpSuLLjIzZp9xBHdeeQL163Zy8Z1/5Z3GXVGXJSISKoVFN31uaoIHZ5/Crn3NXHzHX3llzZaoSxIRCY3CogdOGFvBo1+fwfCBJVx176s89rcPoy5JRCQUCoseGjusP4/84wxOGFfOt3/3Oj9/5m1NQigieUdh0QuG9C/mgWtP5uLjR/OThav4zh+W0dTcGnVZIiK9RqOheklJUYyfXDqNscP687On32bd9r3c+ZXpDOlXHHVpIiI9piuLXmRmfPuzR/GTL01j8XtbueTOv7J2q5ZrFZHcp7AIwSXTD+OBa09m0859fOGOF3ldy7WKSI5TWITk1COG8cjXT6NfSZzL736JBVquVURymMIiRG3LtR4zajDX/9cS7nlhjUZKiUhOUliEbPjAUubMPoWZk0fxg3krufnxeppbNFJKRHKLwqIPlBXHuf3LyeVaH3jpfWb/5xIt1yoiOUVh0Ufalmv9wUVT+LOWaxWRHKOw6GNfOWUc916TXK71ottfZOV6LdcqItlPYRGBs44ewe+vPw13+NIvXuJ5LdcqIllOYRGRSVWDefQbpzFmaH+u/fVifvuKlmsVkeylsIhQ23KtfzdxON9/dDk/fnKllmsVkayksIjYwNIi7vlqDV85ZSx3Pb+Gbz6o5VpFJPtoIsEsUBSPceuFUxg3dAA/enIl63douVYRyS66ssgSZsZ1ZxzOHV9OLtf6hTu0XKuIZA+FRZZpW651934t1yoi2UNhkYW0XKuIZBuFRZZqv1zrf2i5VhGJkMIii6Uu1/rThav4p99ruVYRiYZGQ2W5jpZr/cVVWq5VRPpWqFcWZjbTzN4ys9VmdlMn+1xqZivMrN7MfpvS3mJmrwePx8OsM9u1Ldf600unUfe+lmsVkb4XWliYWRy4HfgcMAm4wswmtdtnIvA9YIa7Twa+nfLyXnc/LnhcEFadueTiE7Rcq4hEI8wri5OA1e6+xt2bgDnAhe32uQ643d23Abj7phDryQvJ5VpnfLxc61NvaLlWEQlfmGExGlibst0QtKU6CjjKzF40s5fNbGbKa2VmVhe0XxRinTnnyBEDefTrMzg2MZh//I2WaxWR8IUZFtZBW/tvtCJgInAmcAVwj5mVB6+Ndfca4MvAz8zsiIM+wGx2ECh1jY2FNc338IGlPHjdJ8u1/usftVyriIQnzLBoAMakbB8GrOtgnz+6+wF3fxd4i2R44O7rgj/XAH8Gjm//Ae5+t7vXuHtNZWVl759BlmtbrvUfzjic/3z5fa57oE7LtYpIKMIMi8XARDObYGYlwOVA+1FNjwFnAZjZcJLdUmvMrMLMSlPaZwArQqw1Z8VixvfOP5YffmEKi97ezKV3vaRZa0Wk14UWFu7eDNwALABWAg+5e72Z3WJmbaObFgBbzGwF8BzwHXffAhwL1JnZ0qD9NndXWBzClSeP42eXHUf9up0sXLEx6nJEJM9YvtwYramp8bq6uqjLiFRLq3PKj59h+tgKfnHV9KjLEZEcYGZLgvvDh6TpPvJIPGbMmprgubc2sUv3LkSkFyks8kxtdYL9za08s1JdUSLSexQWeeaEsRWMGlzGE0vXR12KiOQRhUWeicWMWdUJFq1qZOe+A1GXIyJ5QmGRh2ZVJ2hqaWVhvbqiRKR3KCzy0PFjyhld3o95y9UVJSK9Q2GRh8ySXVEvvN3Ijj3qihKRnlNY5Kna6gQHWpwFKzQrrYj0nMIiT00dPYSxQ/szd5m6okSk5xQWeaqtK+rF1ZvZtrsp6nJEJMcpLPLYrKkJWlqdp+rVFSUiPaOwyGOTqwYzYfgA5i5rPzO8iEjXKCzymFlyrqiX3tnC5l37oy5HRHKYwiLP1U5L0OrwpNbqFpEeUFjkuaNHDuKIygHMU1eUiPSAwiLPmRm11VW88u5WNu3cF3U5IpKjFBYFoLY6gasrSkR6QGFRACaOHMTRIwdpVJSIdJvCokDUVidY/N42NuxQV5SIdJ3CokDMqk4AaCZaEekWhUWBOLxyIJMSgzUqSkS6RWFRQGZVJ3jtg+18uH1v1KWISI5RWBSQ2qArar5mohWRLlJYFJBxwwYwdfQQjYoSkS5TWBSY2uoESxt2sHbrnqhLEZEcorAoMOdPTXZFaVEkEekKhUWBGTO0P8eNKVdXlIh0icKiANVWJ6hft5N3N++OuhQRyREKiwLU1hWl31yISKYUFgWoqrwf08dV6L6FiGRMYVGgaqsTvLnhI1Zv2hV1KSKSAxQWBer8qQnMYJ6uLkQkAwqLAjVycBknjh+qUVEikhGFRQH7fHWCtzftYtXGj6IuRUSynMKigM2ckiBmMHepri5E5NAUFgWsclAppxw+jLnL1+PuUZcjIllMYVHgZlUnWNO4m5Xr1RUlIp1TWBS4mZNHEY8Z85arK0pEOtflsDCzmJkNDqMY6XvDBpZy2hHDmLtMXVEi0rmMwsLMfmtmg81sALACeMvMvpPBcTPN7C0zW21mN3Wyz6VmtsLM6s3styntV5vZ28Hj6kxPSLqutjrB+1v28MaHO6MuRUSyVKZXFpPcfSdwETAfGAtcdagDzCwO3A58DpgEXGFmk9rtMxH4HjDD3ScD3w7ahwI3AycDJwE3m1lFpiclXXPe5FEUxYy56ooSkU5kGhbFZlZMMiz+6O4HgHR9FicBq919jbs3AXOAC9vtcx1wu7tvA3D3TUH7ecBCd98avLYQmJlhrdJF5f1LOH3icOapK0pEOpFpWNwFvAcMABaZ2TggXZ/FaGBtynZD0JbqKOAoM3vRzF42s5ldOBYzm21mdWZW19jYmOGpSEdmTU3QsG0vSxt2RF2KiGShjMLC3f/D3Ue7+/me9D5wVprDrKO3arddBEwEzgSuAO4xs/IMj8Xd73b3GnevqaysTHse0rlzJ4+iOG76gZ6IdCjTG9zfCm5wm5nda2avAZ9Jc1gDMCZl+zCg/TdRA0G3lru/C7xFMjwyOVZ60ZB+xZwxsZL5y9fT2qquKBH5tEy7oa4NbnCfC1QCfw/cluaYxcBEM5tgZiXA5cDj7fZ5jOAKxcyGk+yWWgMsAM41s4rgxva5QZuEqHZagnU79vG3tduiLkVEskymYdHWLXQ+8Ct3X0rHXUUfc/dm4AaSX/IrgYfcvd7MbjGzC4LdFgBbzGwF8BzwHXff4u5bgVtJBs5i4JagTUL02WNHUlIU06JIInIQy2T0i5n9iuQN5gnANCAO/Nndp4dbXuZqamq8rq4u6jJy3uwH6ljasJ2XbjqbWOyQ/z8gInnAzJa4e026/TK9svhvwE3Aie6+Bygh2RUleaZ2WhUbd+6n7n11RYnIJ4oy2cndW83sMODLZgbwvLs/EWplEomzjxlBWXGMucvWcdKEoVGXIyJZItPRULcB3yI51ccK4EYz+3GYhUk0BpQW8ZljRjB/+QZaNCpKRAKZdkOdD5zj7ve5+30kf009K7yyJEqzplaxedd+Xnl3S9SliEiW6Mqss+Upz4f0diGSPT5zzAj6FceZp1FRIhLINCx+DPzNzH5tZvcDS4AfhVeWRKlfSZyzjx3BU29soLmlNepyRCQLZDrdx4PAKcAjweNUd58TZmESrdrqKrbsbuKlNeqKEpE0o6HM7IR2TQ3Bn1VmVuXur4VTlkTtzKMrGVCS7Ir6u4mad0uk0KUbOvuTQ7zmpJ8fSnJUWXGccyaN5Kn6Ddx60RSK41qBV6SQHTIs3D3dzLKSx2ZVV/HY6+t4cfVmzjx6RNTliEiEMvpRnpld3EHzDmB5yoJFkmfOOGo4g8qKmLtsvcJCpMB1ZbqPe4Arg8cvgf8OvGhmh1xeVXJXaVGccyeNYkH9BpqaNSpKpJBlGhatwLHufom7X0JyTe39JNfI/m5YxUn0aqsTfLSvmRfe1kqEIoUs07AY7+4bU7Y3AUcF04Yf6P2yJFvMOHI4Q/oV6wd6IgUuo3sWwAtmNhf4fbD9RZJrcQ8AtodSmWSFkqIY500eyfzlG9h3oIWy4njUJYlIBDK9svgG8CvgOOB44H7gG+6+WyOm8l9tdRW79jezaJW6okQKVaa/4HbgL8CzwNPAIs9k1STJC6ceMYyK/sVaQU+kgGU6RfmlwKsku58uBV4xsy+GWZhkj+J4jJlTEjy9ciP7DrREXY6IRCDTbqh/JrlK3tXu/lXgJOB/hleWZJva6gR7mlp47k39rEakEGUaFrF2P77b0oVjJQ+cPGEowweWMHe5uqJEClGmo6GeMrMFwIPB9mXA/HBKkmxUFI8xc8ooHl7yIXuamulfkulfHRHJB5ne4P4OcDdQDUwD7nZ3/RivwNRWV7H3QAvPrFRXlEihyfh/D939YeDhEGuRLHfi+KFUDipl3rL1fH5aVdTliEgfSreexUckpyI/6CWSI2oHh1KVZKV4zJg1NcGDr37Arv3NDCxVV5RIoThkN5S7D3L3wR08BikoCtOs6gT7m1t5ZuXG9DuLSN7QiCbpkuljKxg1uIwnlmpUlEghUVhIl8RixqzqBItWNbJzn+aQFCkUCgvpslnVCZpaWllYr64okUKhsJAuO35MOaPL+zFPP9ATKRgKC+kys2RX1AtvN7Jjj7qiRAqBwkK6pbY6wYEWZ8GKDVGXIiJ9QGEh3TJ19BDGDu2vactFCoTCQrqlrSvqxdWb2ba7KepyRCRkCgvptllTE7S0Ok/VqytKJN8pLKTbJlcNZsLwAcxTV5RI3lNYSLeZJeeK+us7m9m8a3/U5YhIiBQW0iO10xK0Ojz5hrqiRPKZwkJ65OiRgziicgDzlq2LuhQRCVGoYWFmM83sLTNbbWY3dfD6NWbWaGavB4+vpbzWktL+eJh1SveZGbXVVbzy7lY27dwXdTkiEpLQwsLM4sDtwOeAScAVZjapg11/5+7HBY97Utr3prRfEFad0nO11QlcXVEieS3MK4uTgNXuvsbdm4A5wIUhfp5EZOLIQRw9chBz1RUlkrfCDIvRwNqU7Yagrb1LzGyZmf3BzMaktJeZWZ2ZvWxmF4VYp/SC2uoEi9/bxoYd6ooSyUdhhoV10NZ+idYngPHuXg08Ddyf8tpYd68Bvgz8zMyOOOgDzGYHgVLX2NjYW3VLN8yqTgBoJlqRPBVmWDQAqVcKhwGf6qdw9y3u3jZA/5fA9JTX1gV/rgH+DBzf/gPc/W53r3H3msrKyt6tXrrk8MqBTEoM1qgokTwVZlgsBiaa2QQzKwEuBz41qsnMEimbFwArg/YKMysNng8HZgArQqxVesGs6gSvfbCdD7fvjboUEelloYWFuzcDNwALSIbAQ+5eb2a3mFnb6KYbzazezJYCNwLXBO3HAnVB+3PAbe6usMhytUFX1HxN/yGSd8y9/W2E3FRTU+N1dXVRl1HwPv/zvxAz+OMNp0ddiohkwMyWBPeHD0m/4JZeVVudYGnDDtZu3RN1KSLSixQW0qvOn5rsitKiSCL5RWEhvWrM0P4cN6acecs1KkoknygspNfVVid448OdvLd5d9SliEgvUVhIr/ukK0pXFyL5QmEhva6qvB/Tx1XovoVIHlFYSChqqxO8ueEjVm/aFXUpItILFBYSivOnJjBD63OL5AmFhYRi5OAyThw/VPctRPKEwkJC8/nqBG9v2sWqjR9FXYqI9JDCQkIzc0qCmMHcpbq6EMl1CgsJTeWgUk45fBhzl68nX+YgEylUCgsJ1azqBGsad7NyvbqiRHKZwkJCNXPyKOIx0/QfIjlOYSGhGjawlNOOGMbcZeqKEsllCgsJXW11gve37KF+3c6oSxGRblJYSOjOmzyKopjxhH5zIZKzFBYSuvL+JZw+cTjz1BUlkrMUFtInZk1N0LBtL0sbdkRdioh0g8JC+sS5k0dRHDf9QE8kRykspE8M6VfMGRMrmb98Pa2t6ooSyTUKC+kztdMSrNuxj7+t3RZ1KSLSRQoL6TOfPXYkJUUxLYokkoMUFtJnBpUVc+ZR6ooSyUUKC+lTtdOq2LhzP3XvqytKJJcoLKRPnX3MCMqKY1oUSSTHKCykTw0oLeIzx4xg/vINtKgrSiRnKCykz82aWsXmXft55d0tUZciIhlSWEif+8wxI+hXHGeeRkWJ5AyFhfS5fiVxzj52BE+9sYHmltaoyxGRDCgsJBK11VVs2d3Ey2u2Rl2KiGRAYSGROPPoSgaUxDUqSiRHKCwkEmXFcc6ZNJKn6jdwQF1RIllPYSGRmVVdxfY9B3hx9eaoSxGRNBQWEpkzjhrOoNIizRUlkgMUFhKZ0qI450weyYL6DTQ1qytKJJspLCRSn6+u4qN9zbzwdmPUpYjIISgsJFIzjhzOkH7F+oGeSJYLNSzMbKaZvWVmq83spg5ev8bMGs3s9eDxtZTXrjazt4PH1WHWKdEpKYpx3uSR/GnFRvYdaIm6HBHpRGhhYWZx4Hbgc8Ak4Aozm9TBrr9z9+OCxz3BsUOBm4GTgZOAm82sIqxaJVq11VXs2t/MolXqihLJVmFeWZwErHb3Ne7eBMwBLszw2POAhe6+1d23AQuBmSHVKRE79YhhVPQv1qgokSwWZliMBtambDcEbe1dYmbLzOwPZjami8dKHiiOx5g5JcHTK9UVJZKtwgwL66Ct/QIGTwDj3b0aeBq4vwvHYmazzazOzOoaG9WFkctqqxPsaWrhuTc3RV2KiHQgzLBoAMakbB8GfGoiIHff4u77g81fAtMzPTY4/m53r3H3msrKyl4rXPreyROGMnxgCXOXqytKJBuFGRaLgYlmNsHMSoDLgcdTdzCzRMrmBcDK4PkC4FwzqwhubJ8btEmeKorHmDllFM+u3MSepuaoyxGRdkILC3dvBm4g+SW/EnjI3evN7BYzuyDY7UYzqzezpcCNwDXBsVuBW0kGzmLglqBN8lhtdRV7D7TwrLqiRLKOuefHOsg1NTVeV1cXdRnSAy2tzik/fobpYyv4xVXT0x8gIj1mZkvcvSbdfvoFt2SNeMyYNTXBc29tYtd+dUWJZBOFhWSVWdUJ9je38szKjVGXIiIpFBaSVaaPrWDU4DKeWKpRUSLZRGEhWSUWM86fmmDRqkZ27jsQdTkiElBYSNapnZagqaWVhfXqihLJFkVRFyDS3vFjyhld3o87n3+H1Y27qOhfTHn/Eir6l1DRv5iKAcnnQ/oVE4919GN/EeltCgvJOmbG7DMO5/89t5p7XljDgZaOh3ebweCy4pQw+SRI2gdMef8Shg4oobx/MWXF8T4+I5Hcp7CQrHT1aeO5+rTxuDu79jezfc8Btu1pYtueA2zf08S23cnnqW2Nu/azauMutu9pYndT5xMS9iuOfxImA4IgSQ2YAQdfyQwqLcJMVzFSuBQWktXMjEFlxQwqK2bM0P4ZH7e/ueWTgNkdBMzH2ymhs6eJ9dt3sm1PE9v3HqCz36gWxYzylCuYtoApH1Dc4ZVMxYASyvsVUxTv2W1Bd6el1Wlxp7UVWoLt1ra2lPbW1tR9PWXflOMOOp7Oj3OnpZVPtbn7xzN6pv6zSv1xb4ev0/G+nz7Xtn39oLaD3yO1veP9O6ytk/fLdaMGl3HpiWPS79gDCgvJS6VFcUYOjjNycFnGx7S0Ojv3truCSQmVrbs/eb526x6WNWxn254DNDW3dvqeg8qKqOhfQlHcPvmCbuXjL+ePw6A1+cXd/ks7TyZYkJAdN6ZcYSHSV+IxS97zGFCS8THuzp6mluSVyUFdZW3bTTS3OnEz4jEjZkY8Rsrz5J9t7bGYtdv308cd3GbErLP2traU182Sn/Hx59LJZ7Xty8fPLdi/TWrXXGonXVuzpbbawa8ffJx1+l7t36+zXsGOPjvd50l6CguRHjAzBpQWMaC0iMO08K/kMf3OQkRE0lJYiIhIWgoLERFJS2EhIiJpKSxERCQthYWIiKSlsBARkbQUFiIikpZ1Nk9LrjGzRuD9HrzFcGBzL5UTpXw5D9C5ZKt8OZd8OQ/o2bmMc/fKdDvlTVj0lJnVuXtN1HX0VL6cB+hcslW+nEu+nAf0zbmoG0pERNJSWIiISFoKi0/cHXUBvSRfzgN0LtkqX84lX84D+uBcdM9CRETS0pWFiIikpbAImNmtZrbMzF43sz+ZWVXUNXWXmf27mb0ZnM+jZlYedU3dZWZfMrN6M2s1s5wbuWJmM83sLTNbbWY3RV1PT5jZfWa2yczeiLqWnjCzMWb2nJmtDP5ufSvqmrrLzMrM7FUzWxqcy7+F9lnqhkoys8HuvjN4fiMwyd2vj7isbjGzc4Fn3b3ZzP43gLt/N+KyusXMjgVagbuAf3L3uohLypiZxYFVwDlAA7AYuMLdV0RaWDeZ2RnALuABd58SdT3dZWYJIOHur5nZIGAJcFEu/nux5FJ/A9x9l5kVA38BvuXuL/f2Z+nKItAWFIEB5PB67u7+J3dvDjZfBg6Lsp6ecPeV7v5W1HV000nAandf4+5NwBzgwohr6jZ3XwRsjbqOnnL39e7+WvD8I2AlMDraqrrHk3YFm8XBI5TvLoVFCjP7oZmtBa4E/jXqenrJtcCTURdRoEYDa1O2G8jRL6V8ZWbjgeOBV6KtpPvMLG5mrwObgIXuHsq5FFRYmNnTZvZGB48LAdz9n919DPAb4IZoqz20dOcS7PPPQDPJ88lamZxLjrIO2nL2ijXfmNlA4GHg2+16FnKKu7e4+3EkexBOMrNQugiLwnjTbOXun81w198C84CbQyynR9Kdi5ldDdQCZ3uW35jqwr+XXNMAjEnZPgxYF1EtkiLo338Y+I27PxJ1Pb3B3beb2Z+BmUCvD0IoqCuLQzGziSmbFwBvRlVLT5nZTOC7wAXuvifqegrYYmCimU0wsxLgcuDxiGsqeMFN4XuBle7+06jr6Qkzq2wb7Whm/YDPEtJ3l0ZDBczsYeBokiNv3geud/cPo62qe8xsNVAKbAmaXs7hkV1fAH4OVALbgdfd/bxoq8qcmZ0P/AyIA/e5+w8jLqnbzOxB4EySM5xuBG5293sjLaobzOx04AVgOcn/3gG+7+7zo6uqe8ysGrif5N+vGPCQu98SymcpLEREJB11Q4mISFoKCxERSUthISIiaSksREQkLYWFiIikpbAQ6QIz25V+r0Me/wczOzx4PtDM7jKzd4IZQxeZ2clmVhI8L6gfzUp2U1iI9BEzmwzE3X1N0HQPyYn5Jrr7ZOAaYHgw6eAzwGWRFCrSAYWFSDdY0r8Hc1gtN7PLgvaYmd0RXCnMNbP5ZvbF4LArgT8G+x0BnAz8i7u3AgSz084L9n0s2F8kK+gyV6R7LgaOA6aR/EXzYjNbBMwAxgNTgREkp7++LzhmBvBg8HwyyV+jt3Ty/m8AJ4ZSuUg36MpCpHtOBx4MZvzcCDxP8sv9dOD37t7q7huA51KOSQCNmbx5ECJNweI8IpFTWIh0T0fTjx+qHWAvUBY8rwemmdmh/hssBfZ1ozaRXqewEOmeRcBlwcIzlcAZwKskl7W8JLh3MZLkxHttVgJHArj7O0Ad8G/BLKiY2cS2NTzMbBjQ6O4H+uqERA5FYSHSPY8Cy4ClwLPA/wi6nR4muY7FGyTXDX8F2BEcM49Ph8fXgFHAajNbDvyST9a7OAvIuVlQJX9p1lmRXmZmA919V3B18Coww903BOsNPBdsd3Zju+09HgG+l8Prj0ue0Wgokd43N1iQpgS4NbjiwN33mtnNJNfh/qCzg4OFkh5TUEg20ZWFiIikpXsWIiKSlsJCRETSUliIiEhaCgsREUlLYSFsunrYAAAAD0lEQVQiImkpLEREJK3/DwnGD6AiQCeMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# scores_：dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "Cs = [1e-3, 1e-2, 1e-1, 1, 10, 100, 1000]\n",
    "n_Cs = len(Cs)\n",
    "n_classes = 1\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "\n",
    "for j in range(n_classes):\n",
    "        scores[j][:] = np.mean(lrcv_L1.scores_[j+1],axis = 0)\n",
    "    \n",
    "logloss_mean = -np.mean(scores, axis = 0)\n",
    "plt.plot(np.log10(Cs), logloss_mean.reshape(n_Cs,1)) \n",
    "#plt(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "plt.xlabel('log(C)')\n",
    "plt.ylabel('logloss')\n",
    "plt.show()\n",
    "\n",
    "#print ('C is:',lr_cv.C_)  #对多类分类问题，每个类别的分类器有一个C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.69314718, 0.64684744, 0.48822081, 0.48421281, 0.48482474,\n",
       "       0.48489951, 0.48490597])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logloss_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 0.47483061213583666)\n"
     ]
    }
   ],
   "source": [
    "best_C = np.argmin(logloss_mean)\n",
    "best_score = np.min(logloss_mean)\n",
    "print (Cs[best_C], best_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "得到的结果虽然与Gridcv有区别但是区别不大"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GridSearchCV对正确率超参数优化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "\n",
    "#训练数据多，C可以大一点（更多相信数据）\n",
    "Cs = [ 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression()\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=3, scoring='accuracy')\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7721354166666666\n",
      "{'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEKCAYAAAA1qaOTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4FfW9x/H3NyEQkC3sUaBhCTsSJFWsgiCLVFDQimKrxVaKtrW1ttrSa1fbe0t7r9beaovoVWkfi1oVRSzIIotat6AgKELAokWCIMomRAL53j9mgoeY5RwyJyeQz+t5znNmzvxm5nuGJB9m+425OyIiIjWVluoCRETkxKBAERGRSChQREQkEgoUERGJhAJFREQioUAREZFIKFBERCQSChQREYlESgLFzFqZ2SIzKwzfsypoM9zMVsW8is1sQrk2fzSzfbVXuYiIVMZScae8mf0O+NDdp5vZNCDL3X9URftWwEago7vvDz/LB64HLnL3pvGst02bNp6Tk1Pj+kVE6pOVK1d+4O5tq2uXqkBZDwxz9yIzywaWuXvPKtpPBc5x96+E4+nAYuDLQGG8gZKfn+8FBQU1/wIiIvWIma109/zq2qXqHEp7dy8CCN/bVdN+EjA7Zvw6YG7ZMkREJPUaJGvBZrYY6FDBpJsTXE420B94Ohw/GZgIDItz/qnAVIDOnTsnsmoREUlA0gLF3UdWNs3M3jez7JhDXturWNSlwBx3LwnHBwLdgY1mBtDEzDa6e/dK6pgJzITgkNcxfBUREYlD0gKlGnOBycD08P2JKtpeDvy4bMTdnyJmz8fM9lUWJiIilSkpKWHLli0UFxenupQ6IzMzk44dO5KRkXFM86cqUKYDD5vZ1cC7BIewyq7cutbdp4TjOUAnYHlqyhSRE9WWLVto1qwZOTk5hEc76jV3Z+fOnWzZsoUuXboc0zJSEijuvhMYUcHnBcCUmPHNwCnVLCuuK7xERGIVFxcrTGKYGa1bt2bHjh3HvAzdKS8i9ZbC5Gg13R4KFBGROF121wtcdtcLqS6jzlKgxOO+scFLRCRCTZt+esR+zJgxtGzZknHjxlXY9tvf/jZ5eXn06dOHxo0bk5eXR15eHo888khC63z11VdZsGBBjequTKpOyouISIybbrqJ/fv3c9ddd1U4/c477wRg8+bNjBs3jlWrVh3Tel599VXWrl3LmDFjjrnWymgPRUSkDhgxYgTNmjU7pnkLCws577zzGDRoEEOHDmXDhg0APPjgg/Tr148BAwYwfPhwDhw4wC233MIDDzxwTHs31dEeiojUe7988g3e3Lqn2nZvFgVt4jmP0ufk5vz8gr41ri0eU6dO5Z577qFbt248//zzXHfddSxcuJBf/vKXLFu2jPbt27Nr1y4aN27Mz372M9auXcvtt98eeR0KFBGR49iuXbt48cUX+dKXvnTks0OHDgFw1lln8dWvfpWJEydy8cUXJ70WBYqI1Hvx7kmU7Zk8dM2ZySwnIe5OmzZtKjyncvfdd/PSSy8xb948BgwYwOuvv57UWnQORUTkOJaVlUV2djZz5swBoLS0lNWrVwPw9ttvM3jwYH71q1+RlZXFe++9R7Nmzdi7d29SalGgiIjUAUOGDGHixIksWbKEjh078vTTT8c974MPPsiMGTMYMGAAffv2Zd68eQDccMMN9O/fn/79+zNy5Ej69evHueeey+rVqxk4cKBOyouInCj27fv0CebPPvtsXPPk5OSwdu3aoz7r2rVrhQE0d+7cz3zWtm1bkvWgQQVKHN4o2g1A7VyvISJ1VV06d1IX6ZCXiIhEQoEiIiKRUKCIiEgkFCgiIhIJBYqISLzU83iVFCgiIilS1n39qlWrOPPMM+nbty+nnnoqDz300Gfaqvt6ERGpVpMmTfjLX/5Cbm4uW7duZdCgQZx33nm0bNnySBt1Xy8iItXq0aMHubm5AJx88sm0a9cuoWe7q/t6EZG6Yv402Lam+nbbws4V4zmP0qE/fHF6wqW8/PLLHDx4kG7dusU9j7qvFxGRoxQVFXHllVcya9Ys0tLiO4Ck7uvlxFb2v7evPZXaOkTiFe+eRBJ/tvfs2cPYsWP59a9/zeDBg+OeT93Xi4jIEQcPHuSiiy46sjeRCHVfLyIiRzz88MOsWLGC+++//8jlwIlcxaXu60VE6rmy7uuvuOIKrrjiirjmUff1IiInAp0XrJIOeYmISCQUKCJSb7l7qkuoU2q6PVISKGbWyswWmVlh+J5VQZvhZrYq5lVsZhPCafeb2b9ipuXV/rcQkeNZZmYmO3fuVKiE3J2dO3eSmZl5zMtI1TmUacASd59uZtPC8R/FNnD3pUAeBAEEbAQWxjS5yd2jvURBROqNjh07smXLloS6ODnRZWZm0rFjx2OeP1WBMh4YFg7PApZRLlDKuQSY7+77k1uWiNQXGRkZdOnSJdVlnFBSdQ6lvbsXAYTv7appPwmYXe6z/zSz183s92bWKBlFiohI/JK2h2Jmi4EOFUy6OcHlZAP9gdiLrH8MbAMaAjMJ9m5uqWT+qcBUgM6dOyeyahERSUDSAsXdR1Y2zczeN7Nsdy8KA2N7FYu6FJjj7iUxyy4KBz8xs/uAG6uoYyZB6JCfn6+zb1LnvPFfZwPQ9z+eS3Elxwdtr8TU5vZK1SGvucDkcHgy8EQVbS+n3OGuMIQwMwMmAGsrmE9ERGpRqgJlOjDKzAqBUeE4ZpZvZveUNTKzHKATsLzc/A+Y2RpgDdAG+HUt1CwiIlVIyVVe7r4TGFHB5wXAlJjxzcApFbQ7N5n1iYhI4nSnvIiIREKBIiIikVCgiIhIJBQoIiISCQWKiIhEQoEiIiKRUKCIiEgk9AhgidwbRbsB6JviOkSkdmkPRUREIqFAERGRSChQREQkEgoUERGJhAJFREQioUAREZFIKFBERCQSChQREYmEAkVERCKhQBERkUgoUEREJBIKFBERiYQCRUREIqFAERGRSChQREQkEgoUERGJhAJFREQioUAREZFIKFBERCQSChQREYmEAkVERCKRkkAxs1ZmtsjMCsP3rAraDDezVTGvYjObEE4zM/tPM9tgZuvM7Lu1/y1ERCRWqvZQpgFL3D0XWBKOH8Xdl7p7nrvnAecC+4GF4eSrgE5AL3fvDTxYK1WLiEilUhUo44FZ4fAsYEI17S8B5rv7/nD8m8At7l4K4O7bk1KliIjELVWB0t7diwDC93bVtJ8EzI4Z7wZcZmYFZjbfzHIrm9HMpobtCnbs2FHjwkVEpGINkrVgM1sMdKhg0s0JLicb6A88HfNxI6DY3fPN7GLgXmBIRfO7+0xgJkB+fr4nsm4REYlf0gLF3UdWNs3M3jezbHcvCgOjqkNWlwJz3L0k5rMtwKPh8BzgvhoXLCIiNZKqQ15zgcnh8GTgiSraXs7Rh7sAHic4UQ9wDrAh0upERCRhqQqU6cAoMysERoXjmFm+md1T1sjMcgiu5lpewfxfMrM1wG+AKbVQs4iIVCFph7yq4u47gREVfF5ATDi4+2bglAra7QLGJrHEo+z1TJrwSW2tTkTkuKQ75ePwxwNjuO7jq5m/pojSUp3XFxGpiAIlDkMz1lGK8c0HXuWCO57jmbfex13BIiISS4ESh7Mz1vOnk+7htksHsLf4EF+/v4CL//xPnt/4gYJFRCSkQIlTujkXn9aRJT84h99c3J9tu4v5yj0vcfndL/LK5g9TXZ6ISMopUBKUkZ7G5ad3ZumNw/jFBX3YuP1jJs54gcn3vszrW3alujwRkZSJK1DM7FEzG2tmCqBQZkY6V53VhWd/OJwff7EXr2/ZxYV3PM/UvxTw1rY9qS5PRKTWxRsQfwa+DBSa2XQz65XEmo4rjRumc8053Vjxw+F8f1QPXti0ky/+4Vm+M/s1Nu3Yl+ryRERqTVyB4u6L3f0rwGnAZmCRmf3TzL5mZhnJLPB40Swzg++OyOXZHw3nW8O6sWTd+4y6bTk3/n01//5wf/ULEBE5zsV9CMvMWhM8h2QK8BrwB4KAWZSUyo5TLZs05KbzerHih8P5+lldeHL1Vob/zzJunrOGot0HUl2eiEjSxHWnvJk9BvQC/gpcUNb1PPCQmRUkq7i6oqEXczjBTgXaNG3ET8b1YcqQrty5dCMPvvIuf1+5hSvO+BzfHNaNts0aJalaEZHUiPev5B3u/kxFE9w9P8J66qTsQ0Wc5Pvg/nHQ+0LoPQ6anxzXvB1aZPKrCf2YOrQrf3ymkFkvbGb2y+9y1Vk5XDO0Ky2bNExu8SIitSTeQ169zaxl2YiZZZnZt5JUU52zLb0DH6S1hY93wPyb4LbecM9IeP4P8OG/4lpGp1ZN+N0lA1h0w1BG923PjOWbGPLbpdy+eAN7i0uqX4CISB0Xb6B8I+yQEQB3/wj4RnJKqns+SWvM9gYd4NsvwbdfgXN/CocPwqKfwf/mwYyzYfnvYPtbUM2d813bNuUPkway4PqhnNW9DbcvLmTI75by52Wb2H/wUC19IxGR6MV7yCvNzMzDfkbMLB2on8dq2vaAtjfC0Bvho3dg3ZPBa+l/wdL/hNa50OdC6H0BZOeBWYWL6dmhGTOuHMSaLbu5bdF6frvgLf7vubf51rDufPmMzmRmpNfyFxMRqZl4A+Vp4GEzmwE4cC2wIGlVHS+yPgdfuC547d0Gb82DN+fCc7fDs7dCi85BsPS5EDqeDmmf3SHs37EF933tdFa+8yG3LtzALfPeZOaKt/nOiO5MHNSJhg10L6mIHB/iDZQfAdcA3wQMWAjcU+Uc9U2zDvD5KcFr/4ew/h/Bnssrd8OLd0LT9tBrXBAwOWdD+tG37wz6XCv+9o3B/HPTB9y6cAM3z1nLjOWbuH5EDybknUyDdAWLiNRtcQWKu5cS3C3/5+SWc4Jo0goGXhG8ivdA4UJYNxdWz4aC/4PGWdDz/CBcug6HjMwjs36hWxvOvLY1yzbs4NaF67nx76v507KN3DCyB2P7Z5OWVvEhNBGRVIv3PpRcgkft9gGO/PVz965JquvEkdkc+l8SvEoOwMYl4XmXebDqAWjYFHJHB4fFuo+CRk0xM4b3bMewHm15+o33uW3Rer4z+zXuXLqR74/qwag+7bFKzs2IiKRKvIe87gN+DvweGA58jeDQlyQio3FwD0vvcXDoIGxeEZxzeespeOMxaJAJ3UYEey49x2CNsxjTrwOj+rRn3utbuX1xIVP/upJTO7bgB6N7MjS3jYJFROqMeAOlsbsvCa/0egf4hZk9SxAyciwaNITuI4PXuN/Duy98esXY+qcgrQF0GQq9LyS91zjG553C2P7ZPPbae/xhcSGT732Zz+dk8YPRPRnctXWqv42ISNyBUhx2XV9oZtcB7wHtkldWPZOWHpyozzkbzvsNbH0N1j0R7L3M+x489X3ofCYNel/Apb0vYELeMB4q+Dd3PFPIpJkvcnb3Nnx/dA9O65yV6m8iIvVYvJcOfQ9oAnwXGARcAUxOVlH1WloadBwEo26B774G1z4PQ38IBz6CBdPg931peO8Irjz0GCuu7sxPxvZmXdEeLv7TP7n6/ldY+97uVH8DEamnqt1DCW9ivNTdbwL2EZw/kdpgBh36Ba/hP4YPNgZXi617Ehb/gkaLf8GUdn356pljefTAaUxf+SHj/vgc5/fvwA0je5Dbvlmqv4GI1CPVBoq7HzazQbF3ykuKtOkOQ74fvHb9+8iNlA2f+28ux7k0qwsrmwzhtg29GL22iAl5Hbl+RC45bU5KdeUiUg/Eew7lNeAJM/s78HHZh+7+WFKqkuq17ASDvxm89m2Ht+aRvu5JTv/X33jQDrG7WXueeGMg014/nS4Dz+W6kb04pWXjVFctIieweAOlFbATODfmMwcUKHVB03aQ//Xgtf9D2LCAFuue5MqNS/hq2gI+WPMHFq/O52CPsYwZN5F2Wc1TXbGInIDivVNe502OF01aQd6XIe/L2Cd7oXARJ73+OBdtXEijTUvYfftPWdNmKJ8bMonmfc8L7o0REYlAvHfK30ewR3IUd/965BVJdBo1g34X07jfxVBSzI7VC/jXs7Pp8cEKmj8+n4NzG0PuaBr2uxB6nBe0FxE5RvEe8poXM5wJXARsPdaVmlkr4CEgB9hMcBXZR+XaDCe4M79ML2CSuz8e3lRZ9tevHfCyu0841nrqhYxM2uZPoG3+BDYWfchf5z1Cq3cWMGb9MlqvfwJPb4R1Gx48kbLnF4M9HRGRBMR7yOvR2HEzmw0srsF6pwFL3H26mU0Lx39Ubp1Lgbxwfa2AjQS9HOPuQ2JqeRR4oga11Dvds1vxnW9M5c2tk5i2cB271j/HhMyVXPjOSpptWACWDl2GBF3A9LoAmrVPdckichyIdw+lvFygcw3WOx4YFg7PApZRLlDKuQSY7+77Yz80s2YEFwroHM8x6HNyc+6+6gxW/bsnty48m5sLd3DOSVu4sfN6+u5aTtpTP4CnboROZwSdV/YaFzwDRkSkAvGeQ9nL0edQtlF1AFSnvbsXAbh7kZlV143LJOC2Cj6/iGBPZ08Naqn38jq15K9Xn8FLb+/k1oUbuGBdJ05pMY6fDoVR9grpbz0JT/9H8MrOCx8aNh7a5Ka6dBGpQ+I95JXw2VozWwx0qGDSzQkuJxvoT/DUyPIup5oHfZnZVGAqQOfONdmpOvGd0bU1D10zmOc2fsD/LNzAtQt3kdP6DL438kou6FQcBMu6J+GZXwWvtr2Ccy69L4AO/St93LGI1A/x7qFcBDzj7rvD8ZbAMHd/vLJ53H1kFct738yyw72TbGB7Fau/FJjj7iXlltEaOJ1gL6VS7j4TmAmQn5+vO/2rYWYMyW3L2d3bsGTddm5dtIHvPbSKO9s15fujLuO8q68nbe97QZf7b86FZ/8HVvwOsnKCYOk9HtwVLiL1kMXTm4qZrXL3vHKfvebuA49ppWb/DeyMOSnfyt1/WEnbF4EfhyfpYz+/FjjT3ePupDI/P98LCgoSrveN/zobgL7/8VzC8x7vSkud+Wu3cdui9Wza8TF9T27OD0b3YHjPdsGzWPbtCLrbX/ckvL0cSksooQH7006iRZ9K/09RbxwudUoOl1JyuOz9s8Mff/BvDpNGwxYV7dBLeQd3bwPQ9opT2fbqds3faHdKl2NahpmtdPf86trFe1K+ol6Jj/WEPsB04GEzuxp4F5gIYGb5wLXuPiUczwE6AcsrWMakcDmSRGlpxthTsxnTrwNPrHqP2xcX8vX7CxjYuSU3ju7JF7q1wQZdBYOuggO7YMPTHHj8RjJLD0DR6lSXX2MOlLpTWlr27pS6c9grGA+HS8sNVyYdyDA4iVLMnLR9u2rrax3X3A4BYNpecSnbXiWfFCd9XfHuodwL7ALuJPgd+w6Q5e5XJbW6iGkPpeZKDpfyyMot/O+SQop2FzO4aytuHN2T/JxP71upa9uruOQwe4pL2HOghN0HDoXvJewpLmH3/vD9QAl7Dhz69PMDQfu9nxyiql+RNIPmjTNonplBi8YZNG/cIHg/Mh6+MhscGS+b3rxxAxo1SK9z26uu0/ZKTBTbK+o9lO8APyW4GRGC+0F+coy1yXEsIz2Ny0/vzEUDT+HBl9/ljqWbuGTGC5zToy0/GN2DUzu2jHydpaXO3k9iguBA1SEQjB86Mv7JodIql984I/2oIOjQPJMe7ZuF4w2OhEKLCoLjpIYNSEvT+SIRiP8qr48Jbj4UASAzI52rzurCZZ/vzF9e2MyM5Zu48I7nGd2nPeMPtyUnfcdR7YtLDlceBBHtJcT+we/QIjNmTyA2ED67p9CwQbzPmRORqsR7ldciYKK77wrHs4AH3f28ZBYndV/jhulcc043vnxGZ+57fjN3r3ibRZ9cTbe0bfity47sKRyMYy8h9n/+HZpn0rN9syoPGbVoEnzetFGD4AIBEUmpeA95tSkLEwB3/yiOmxGlHmmWmcF3R+Qy+cwcfjP956w/fDInd2hO88YNyp1j+OyegvYSRE4M8QZKqZl1dvd34cjVV7qnQz6jRZMMrsxcAUDfr1yX4mpEpDbFGyg3A8+ZWdnlu0MJ7z4XERGB+E/KLwjvEZkKrCLo3fdAMgsTEZHjS7wn5acA1wMdCQJlMPACRz8SWERE6rF4z4ReD3weeMfdhwMDgR1VzyIiIvVJvIFS7O7FAGbWyN3fAnomrywRETnexHtSfkvYw/DjwCIz+4gaPAJYREROPPGelC/rIv4XZrYUaAEsSFpVIiJy3Em4x2B3r6jnXxERqed0e7KIiERCgSIiIpFQoIiISCQUKCIiEgkFioiIREKBIiIikVCgiIhIJBQoIiISCQWKiIhEQoEiIiKRUKCIiEgkFCgiIhIJBYqIiERCgSIiIpFQoIiISCQUKCIiEomUBIqZtTKzRWZWGL5nVdBmuJmtinkVm9mEcNoIM3s1/Pw5M+te+99CRERipWoPZRqwxN1zgSXh+FHcfam757l7HnAusB9YGE7+M/CVcNrfgJ8ks9i+2S3om90imasQETnupSpQxgOzwuFZwIRq2l8CzHf3/eG4A83D4RbA1sgrFBGRhCT8TPmItHf3IgB3LzKzdtW0nwTcFjM+BfiHmR0A9gCDk1OmiIjEK2mBYmaLgQ4VTLo5weVkA/2Bp2M+vgE4391fMrObCMJmSiXzTwWmAnTu3DmRVYuISAKSFijuPrKyaWb2vpllh3sn2cD2KhZ1KTDH3UvCedsCA9z9pXD6Q8CCKuqYCcwEyM/P9wS/hoiIxClV51DmApPD4cnAE1W0vRyYHTP+EdDCzHqE46OAdZFXKCIiCUnVOZTpwMNmdjXwLjARwMzygWvdfUo4ngN0ApaXzejuh8zsG8CjZlZKEDBfr9XqRUTkM1ISKO6+ExhRwecFxJwLcffNwCkVtJsDzEliiSIikiDdKS8iIpFQoIiISCQUKCIiEgkFioiIREKBIiIikVCgiIhIJBQoIiISCQWKiIhEQoEiIiKRUKCIiEgkFCgiIhIJBYqIiERCgSIiIpFQoIiISCQUKCIiEgkFioiIREKBIiIikVCgiIhIJBQoIiISCQWKiIhEokGqCzgufO2pVFcgIlLnaQ9FREQioUAREZFIKFBERCQSChQREYmEAkVERCKhQBERkUgoUEREJBIKFBERiURKAsXMWpnZIjMrDN+zKmgz3MxWxbyKzWxCOO1cM3vVzNaa2Swz0w2aIiIplqo9lGnAEnfPBZaE40dx96XunufuecC5wH5goZmlAbOASe7eD3gHmFx7pYuISEVSFSjjCUKB8H1CNe0vAea7+36gNfCJu28Ipy0CvpSUKkVEJG6pCpT27l4EEL63q6b9JGB2OPwBkGFm+eH4JUCnymY0s6lmVmBmBTt27Khh2SIiUpmknXsws8VAhwom3ZzgcrKB/sDTAO7uZjYJ+L2ZNQIWAocqm9/dZwIzAfLz8z2RdYuISPySFijuPrKyaWb2vpllu3tRGBjbq1jUpcAcdy+JWfYLwJBwWaOBHhGVLSIixyhVh7zm8umJ9MnAE1W0vZxPD3cBYGbtwvdGwI+AGUmoUUREEpCqQJkOjDKzQmBUOI6Z5ZvZPWWNzCyH4PzI8nLz32Rm64DXgSfd/ZnaKFri0ze7BX2zW6S6DBGpZeZef04r5Ofne0FBQarLOPHdNzZ414PJRE4IZrbS3fOra6c75UVEJBIKFBERiYQCRUREIqFAERGRSChQREQkEgoUERGJhAJFREQioUAREZFIKFBERCQSChQREYmEAkVERCKhZ7FL9NSHl0i9pD0UERGJhAJFREQioUAREZFIKFBERCQSChQREYmEAkVERCKhQBERkUgoUEREJBIKFBERiYS5e6prqDVmtgN45xhnbwN8EGE5UVFdiVFdiVFdiTlR6/qcu7etrlG9CpSaMLMCd89PdR3lqa7EqK7EqK7E1Pe6dMhLREQioUAREZFIKFDiNzPVBVRCdSVGdSVGdSWmXtelcygiIhIJ7aGIiEgkFCiVMLOJZvaGmZWaWaVXR5jZGDNbb2YbzWxaLdTVyswWmVlh+J5VSbvDZrYqfM1NYj1Vfn8za2RmD4XTXzKznGTVkmBdV5nZjphtNKUWarrXzLab2dpKppuZ/W9Y8+tmdlqya4qzrmFmtjtmW/2slurqZGZLzWxd+Lt4fQVtan2bxVlXrW8zM8s0s5fNbHVY1y8raJPc30d316uCF9Ab6AksA/IraZMObAK6Ag2B1UCfJNf1O2BaODwN+G0l7fbVwjaq9vsD3wJmhMOTgIfqSF1XAXfU8s/UUOA0YG0l088H5gMGDAZeqiN1DQPm1ea2CtebDZwWDjcDNlTw71jr2yzOump9m4XboGk4nAG8BAwu1yapv4/aQ6mEu69z9/XVNDsd2Ojub7v7QeBBYHySSxsPzAqHZwETkry+qsTz/WPrfQQYYWZWB+qqde6+Aviwiibjgb944EWgpZll14G6UsLdi9z91XB4L7AOOKVcs1rfZnHWVevCbbAvHM0IX+VPkif191GBUjOnAP+OGd9C8n+w2rt7EQQ/2EC7StplmlmBmb1oZskKnXi+/5E27n4I2A20TlI9idQF8KXwMMkjZtYpyTXFIxU/T/E6MzyUMt/M+tb2ysNDMwMJ/tcdK6XbrIq6IAXbzMzSzWwVsB1Y5O6Vbq9k/D42iGpBxyMzWwx0qGDSze7+RDyLqOCzGl82V1VdCSyms7tvNbOuwDNmtsbdN9W0tnLi+f5J2UbViGedTwKz3f0TM7uW4H9t5ya5ruqkYlvF41WCrjf2mdn5wONAbm2t3MyaAo8C33P3PeUnVzBLrWyzaupKyTZz98NAnpm1BOaYWT93jz03ltTtVa8Dxd1H1nARW4DY/9l2BLbWcJlV1mVm75tZtrsXhbv22ytZxtbw/W0zW0bwv6ioAyWe71/WZouZNQBakPzDK9XW5e47Y0bvBn6b5JrikZSfp5qK/WPp7v8wsz+ZWRt3T3qfVWaWQfBH+wF3f6yCJinZZtXVlcptFq5zV/h7PwaIDZSk/j7qkFfNvALkmlkXM2tIcJIraVdUheYCk8PhycBn9qTMLMvMGoXDbYCzgDeTUEs83z+23kuAZzw8I5hE1dZV7jj7hQTHwVNtLvDV8Mpvv0hlAAADE0lEQVSlwcDussObqWRmHcqOs5vZ6QR/N3ZWPVck6zXg/4B17n5bJc1qfZvFU1cqtpmZtQ33TDCzxsBI4K1yzZL7+1ibVyEcTy/gIoI0/wR4H3g6/Pxk4B8x7c4nuMpjE8GhsmTX1RpYAhSG763Cz/OBe8LhLwBrCK5uWgNcncR6PvP9gVuAC8PhTODvwEbgZaBrLf37VVfXb4A3wm20FOhVCzXNBoqAkvBn62rgWuDacLoBd4Y1r6GSqwtTUNd1MdvqReALtVTX2QSHY14HVoWv81O9zeKsq9a3GXAq8FpY11rgZxX83Cf191F3youISCR0yEtERCKhQBERkUgoUEREJBIKFBERiYQCRUREIqFAEYmQme2rvlWV8z8S9m6AmTU1s7vMbFPYe+wKMzvDzBqGw/X6xmSpexQoInVE2N9Turu/HX50D8FdzLnu3pegh+Q2HnR4uQS4LCWFilRCgSKSBOGd2/9tZmvNbI2ZXRZ+nhZ2w/GGmc0zs3+Y2SXhbF8h7PnAzLoBZwA/cfdSCLrRcfenwraPh+1F6gztMoskx8VAHjAAaAO8YmYrCLrByQH6E/QUvQ64N5znLIK71gH6Aqs86OyvImuBzyelcpFjpD0UkeQ4m6A348Pu/j6wnCAAzgb+7u6l7r6NoNuXMtnAjngWHgbNQTNrFnHdIsdMgSKSHJU9tKiqhxkdIOhrCYJ+oAaYWVW/o42A4mOoTSQpFCgiybECuCx84FFbgsfsvgw8R/BgrzQza0/wqNgy64DuAB48u6YA+GVMr7W5ZjY+HG4N7HD3ktr6QiLVUaCIJMccgl5fVwPPAD8MD3E9StCj71rgLoIn/e0O53mKowNmCsGD1jaa2RqC57aUPetjOPCP5H4FkcSot2GRWmZmTT14kl9rgr2Ws9x9W/gMi6XheGUn48uW8RjwY3dfXwsli8RFV3mJ1L554YOQGgK/CvdccPcDZvZzgud+v1vZzOFDwx5XmEhdoz0UERGJhM6hiIhIJBQoIiISCQWKiIhEQoEiIiKRUKCIiEgkFCgiIhKJ/wdanVc9/DGzYgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    #plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'accuracy' )\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
